{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe6e643-9453-4381-9445-bd471685fb96",
   "metadata": {
    "id": "1fe6e643-9453-4381-9445-bd471685fb96"
   },
   "source": [
    "## Exploring the SQUADv2 dataset using Autolabel"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff",
   "metadata": {
    "id": "80110a5b-2b3e-45e2-a2da-f6fa00200dff"
   },
   "source": [
    "#### Setup the API Keys for providers that you want to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "92993c83-4473-4e05-9510-f543b070c7d0",
   "metadata": {
    "executionInfo": {
     "elapsed": 160,
     "status": "ok",
     "timestamp": 1686855643474,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "92993c83-4473-4e05-9510-f543b070c7d0"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# provide your own OpenAI API key here\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c246f85",
   "metadata": {
    "id": "9c246f85"
   },
   "source": [
    "#### Install the autolabel library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bc181e31",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 21972,
     "status": "ok",
     "timestamp": 1686855440303,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "bc181e31",
    "outputId": "e675d713-8425-4e57-d0cc-ccb78e34685b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting refuel-autolabel[openai]\n",
      "  Downloading refuel_autolabel-0.0.3-py3-none-any.whl (57 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m57.4/57.4 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting loguru>=0.5.0 (from refuel-autolabel[openai])\n",
      "  Downloading loguru-0.7.0-py3-none-any.whl (59 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting numpy>=1.23.0 (from refuel-autolabel[openai])\n",
      "  Downloading numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m66.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: requests>=2.27.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.27.1)\n",
      "Collecting datasets>=2.7.0 (from refuel-autolabel[openai])\n",
      "  Downloading datasets-2.13.0-py3-none-any.whl (485 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m485.6/485.6 kB\u001b[0m \u001b[31m34.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting langchain>=0.0.190 (from refuel-autolabel[openai])\n",
      "  Downloading langchain-0.0.201-py3-none-any.whl (1.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m68.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting nervaluate>=0.1.8 (from refuel-autolabel[openai])\n",
      "  Downloading nervaluate-0.1.8-py3-none-any.whl (24 kB)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.5.3)\n",
      "Requirement already satisfied: scikit-learn>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.2.2)\n",
      "Requirement already satisfied: tenacity>=8.2.2 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (8.2.2)\n",
      "Collecting SQLAlchemy==1.4.47 (from refuel-autolabel[openai])\n",
      "  Downloading SQLAlchemy-1.4.47-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m93.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting regex>=2023.6.3 (from refuel-autolabel[openai])\n",
      "  Downloading regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (770 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m770.4/770.4 kB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting rich>=13.3.5 (from refuel-autolabel[openai])\n",
      "  Downloading rich-13.4.2-py3-none-any.whl (239 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m239.4/239.4 kB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scipy>=1.10.1 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (1.10.1)\n",
      "Collecting pydantic>=1.10.9 (from refuel-autolabel[openai])\n",
      "  Downloading pydantic-1.10.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m124.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (2.0.1+cu118)\n",
      "Requirement already satisfied: matplotlib>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from refuel-autolabel[openai]) (3.7.1)\n",
      "Collecting wget>=3.2 (from refuel-autolabel[openai])\n",
      "  Downloading wget-3.2.zip (10 kB)\n",
      "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Collecting openai>=0.27.4 (from refuel-autolabel[openai])\n",
      "  Downloading openai-0.27.8-py3-none-any.whl (73 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.6/73.6 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting tiktoken>=0.3.3 (from refuel-autolabel[openai])\n",
      "  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m101.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy==1.4.47->refuel-autolabel[openai]) (2.0.2)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (9.0.0)\n",
      "Collecting dill<0.3.7,>=0.3.0 (from datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (4.65.0)\n",
      "Collecting xxhash (from datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting multiprocess (from datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (2023.4.0)\n",
      "Collecting aiohttp (from datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m76.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.11.0 (from datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (23.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.7.0->refuel-autolabel[openai]) (6.0)\n",
      "Collecting async-timeout<5.0.0,>=4.0.0 (from langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
      "Collecting dataclasses-json<0.6.0,>=0.5.7 (from langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading dataclasses_json-0.5.8-py3-none-any.whl (26 kB)\n",
      "Collecting langchainplus-sdk>=0.0.9 (from langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading langchainplus_sdk-0.0.10-py3-none-any.whl (21 kB)\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /usr/local/lib/python3.10/dist-packages (from langchain>=0.0.190->refuel-autolabel[openai]) (2.8.4)\n",
      "Collecting openapi-schema-pydantic<2.0,>=1.2 (from langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading openapi_schema_pydantic-1.2.4-py3-none-any.whl (90 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.0/90.0 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.0.7)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (4.39.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (8.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib>=3.5.0->refuel-autolabel[openai]) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->refuel-autolabel[openai]) (2022.7.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10.9->refuel-autolabel[openai]) (4.5.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2022.12.7)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.27.0->refuel-autolabel[openai]) (3.4)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=13.3.5->refuel-autolabel[openai]) (2.14.0)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.0.0->refuel-autolabel[openai]) (3.1.0)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.12.0)\n",
      "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (1.11.1)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (3.1.2)\n",
      "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->refuel-autolabel[openai]) (2.0.0)\n",
      "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->refuel-autolabel[openai]) (3.25.2)\n",
      "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->refuel-autolabel[openai]) (16.0.5)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai]) (23.1.0)\n",
      "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting yarl<2.0,>=1.0 (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m32.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->datasets>=2.7.0->refuel-autolabel[openai])\n",
      "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
      "Collecting marshmallow<4.0.0,>=3.3.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading marshmallow-3.19.0-py3-none-any.whl (49 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.1/49.1 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting marshmallow-enum<2.0.0,>=1.5.1 (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading marshmallow_enum-1.5.1-py2.py3-none-any.whl (4.2 kB)\n",
      "Collecting typing-inspect>=0.4.0 (from dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
      "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=13.3.5->refuel-autolabel[openai]) (0.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib>=3.5.0->refuel-autolabel[openai]) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->refuel-autolabel[openai]) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->refuel-autolabel[openai]) (1.3.0)\n",
      "Collecting mypy-extensions>=0.3.0 (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain>=0.0.190->refuel-autolabel[openai])\n",
      "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
      "Building wheels for collected packages: wget\n",
      "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9657 sha256=c2675fb9e3f669266c4925b33e30e39854864c6d24380f2ad48c712bcd8e0457\n",
      "  Stored in directory: /root/.cache/pip/wheels/8b/f1/7f/5c94f0a7a505ca1c81cd1d9208ae2064675d97582078e6c769\n",
      "Successfully built wget\n",
      "Installing collected packages: wget, nervaluate, xxhash, SQLAlchemy, regex, pydantic, numpy, mypy-extensions, multidict, marshmallow, loguru, frozenlist, dill, async-timeout, yarl, typing-inspect, tiktoken, rich, openapi-schema-pydantic, multiprocess, marshmallow-enum, langchainplus-sdk, huggingface-hub, aiosignal, dataclasses-json, aiohttp, openai, langchain, datasets, refuel-autolabel\n",
      "  Attempting uninstall: SQLAlchemy\n",
      "    Found existing installation: SQLAlchemy 2.0.10\n",
      "    Uninstalling SQLAlchemy-2.0.10:\n",
      "      Successfully uninstalled SQLAlchemy-2.0.10\n",
      "  Attempting uninstall: regex\n",
      "    Found existing installation: regex 2022.10.31\n",
      "    Uninstalling regex-2022.10.31:\n",
      "      Successfully uninstalled regex-2022.10.31\n",
      "  Attempting uninstall: pydantic\n",
      "    Found existing installation: pydantic 1.10.7\n",
      "    Uninstalling pydantic-1.10.7:\n",
      "      Successfully uninstalled pydantic-1.10.7\n",
      "  Attempting uninstall: numpy\n",
      "    Found existing installation: numpy 1.22.4\n",
      "    Uninstalling numpy-1.22.4:\n",
      "      Successfully uninstalled numpy-1.22.4\n",
      "  Attempting uninstall: rich\n",
      "    Found existing installation: rich 13.3.4\n",
      "    Uninstalling rich-13.3.4:\n",
      "      Successfully uninstalled rich-13.3.4\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.24.3 which is incompatible.\n",
      "tensorflow 2.12.0 requires numpy<1.24,>=1.22, but you have numpy 1.24.3 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed SQLAlchemy-1.4.47 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 dataclasses-json-0.5.8 datasets-2.13.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.15.1 langchain-0.0.201 langchainplus-sdk-0.0.10 loguru-0.7.0 marshmallow-3.19.0 marshmallow-enum-1.5.1 multidict-6.0.4 multiprocess-0.70.14 mypy-extensions-1.0.0 nervaluate-0.1.8 numpy-1.24.3 openai-0.27.8 openapi-schema-pydantic-1.2.4 pydantic-1.10.9 refuel-autolabel-0.0.3 regex-2023.6.3 rich-13.4.2 tiktoken-0.4.0 typing-inspect-0.9.0 wget-3.2 xxhash-3.2.0 yarl-1.9.2\n"
     ]
    },
    {
     "data": {
      "application/vnd.colab-display-data+json": {
       "pip_warning": {
        "packages": [
         "numpy"
        ]
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!pip install 'refuel-autolabel[openai]'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2883ca14",
   "metadata": {
    "id": "2883ca14"
   },
   "source": [
    "#### Download the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "16ce0de2",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 12140,
     "status": "ok",
     "timestamp": 1686855487231,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "16ce0de2",
    "outputId": "655b92dc-d6d5-46da-f5a2-9fa644e4a135"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading seed example dataset to \"seed.csv\"...\n",
      "\n",
      "\n",
      "Downloading test dataset to \"test.csv\"...\n"
     ]
    }
   ],
   "source": [
    "from autolabel import get_data\n",
    "\n",
    "get_data(\"squad_v2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b37754b9",
   "metadata": {
    "id": "b37754b9"
   },
   "source": [
    "This downloads two datasets:\n",
    "* `test.csv`: This is the larger dataset we are trying to label using LLMs\n",
    "* `seed.csv`: This is a small dataset where we already have human-provided labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84b014d1-f45c-4479-9acc-0d20870b1786",
   "metadata": {
    "id": "84b014d1-f45c-4479-9acc-0d20870b1786"
   },
   "source": [
    "## Start the labeling process!\n",
    "\n",
    "Labeling with Autolabel is a 3-step process:\n",
    "* First, we specify a labeling configuration (see `config.json` below)\n",
    "* Next, we do a dry-run on our dataset using the LLM specified in `config.json` by running `agent.plan`\n",
    "* Finally, we run the labeling with `agent.run`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c093fe91-3508-4140-8bd6-217034e3cce6",
   "metadata": {
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1686855487232,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "c093fe91-3508-4140-8bd6-217034e3cce6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "from autolabel import LabelingAgent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c93fae0b",
   "metadata": {
    "executionInfo": {
     "elapsed": 124,
     "status": "ok",
     "timestamp": 1686855559810,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "c93fae0b"
   },
   "outputs": [],
   "source": [
    "# load the config\n",
    "with open(\"config_squad_v2.json\") as f:\n",
    "    config = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cb3f033",
   "metadata": {
    "id": "5cb3f033"
   },
   "source": [
    "Let's review the configuration file below. You'll notice the following useful keys:\n",
    "* `task_type`: `question_answering` (since it's a question answering task)\n",
    "* `model`: `{'provider': 'openai', 'name': 'gpt-3.5-turbo'}` (use a specific OpenAI model)\n",
    "* `prompt.task_guidelines`: `'You are an expert at answering questions based on wikipedia articles` (how we describe the task to the LLM)\n",
    "* `prompt.few_shot_num`: 3 (how many labeled examples to provide to the LLM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e7297a74",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 248,
     "status": "ok",
     "timestamp": 1686855564540,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "e7297a74",
    "outputId": "42563607-94e9-40f7-cd03-575a8df65cc1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'task_name': 'OpenbookQAWikipedia',\n",
       " 'task_type': 'question_answering',\n",
       " 'dataset': {'label_column': 'answer', 'delimiter': ','},\n",
       " 'model': {'provider': 'openai', 'name': 'gpt-3.5-turbo'},\n",
       " 'prompt': {'task_guidelines': 'You are an expert at answering questions based on wikipedia articles. Your job is to answer the following questions using the context provided with the question. The answer is a continuous span of words from the context. Use the context to answer the question. If the question cannot be answered using the context, answer the question as unanswerable.',\n",
       "  'few_shot_examples': [{'question': 'What was created by the modern Conservative Party in 1859 to define basic Conservative principles?',\n",
       "    'answer': 'unanswerable',\n",
       "    'context': \"The modern Conservative Party was created out of the 'Pittite' Tories of the early 19th century. In the late 1820s disputes over political reform broke up this grouping. A government led by the Duke of Wellington collapsed amidst dire election results. Following this disaster Robert Peel set about assembling a new coalition of forces. Peel issued the Tamworth Manifesto in 1834 which set out the basic principles of Conservatism; – the necessity in specific cases of reform in order to survive, but an opposition to unnecessary change, that could lead to 'a perpetual vortex of agitation'. Meanwhile, the Whigs, along with free trade Tory followers of Robert Peel, and independent Radicals, formed the Liberal Party under Lord Palmerston in 1859, and transformed into a party of the growing urban middle-class, under the long leadership of William Ewart Gladstone.\"},\n",
       "   {'question': 'When is King Mom symbolically burnt?',\n",
       "    'answer': 'On the evening before Lent',\n",
       "    'context': \"Carnival means weeks of events that bring colourfully decorated floats, contagiously throbbing music, luxuriously costumed groups of celebrants of all ages, King and Queen elections, electrifying jump-ups and torchlight parades, the Jouvert morning: the Children's Parades and finally the Grand Parade. Aruba's biggest celebration is a month-long affair consisting of festive 'jump-ups' (street parades), spectacular parades and creative contests. Music and flamboyant costumes play a central role, from the Queen elections to the Grand Parade. Street parades continue in various districts throughout the month, with brass band, steel drum and roadmarch tunes. On the evening before Lent, Carnival ends with the symbolic burning of King Momo.\"},\n",
       "   {'question': 'How far does the Alps range stretch?',\n",
       "    'answer': 'the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland',\n",
       "    'context': 'The Alps are a crescent shaped geographic feature of central Europe that ranges in a 800 km (500 mi) arc from east to west and is 200 km (120 mi) in width. The mean height of the mountain peaks is 2.5 km (1.6 mi). The range stretches from the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland. The range continues toward Vienna in Austria, and east to the Adriatic Sea and into Slovenia. To the south it dips into northern Italy and to the north extends to the south border of Bavaria in Germany. In areas like Chiasso, Switzerland, and Neuschwanstein, Bavaria, the demarcation between the mountain range and the flatlands are clear; in other places such as Geneva, the demarcation is less clear. The countries with the greatest alpine territory are Switzerland, France, Austria and Italy.'}],\n",
       "  'few_shot_selection': 'fixed',\n",
       "  'few_shot_num': 3,\n",
       "  'example_template': 'Context: {context}\\nQuestion: {question}\\nAnswer: {answer}'}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97ce9a02",
   "metadata": {
    "executionInfo": {
     "elapsed": 113,
     "status": "ok",
     "timestamp": 1686855649734,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "97ce9a02"
   },
   "outputs": [],
   "source": [
    "# create an agent for labeling\n",
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "982b96e6",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "92667a39",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 580,
     "referenced_widgets": [
      "836e2c02160c4f03903eb0583bb9d700",
      "c937612e207b4f80996a0f5265672f59"
     ]
    },
    "executionInfo": {
     "elapsed": 1964,
     "status": "ok",
     "timestamp": 1686855655890,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "92667a39",
    "outputId": "b4c947f7-d63a-4118-cac0-8ef7feaca947"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "836e2c02160c4f03903eb0583bb9d700",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $7.5646 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0038 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$7.5646\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0038\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at answering questions based on wikipedia articles. Your job is to answer the following questions using the context provided with the question. The answer is a continuous span of words from the context. Use the context to answer the question. If the question cannot be answered using the context, answer the question as unanswerable.\n",
      "\n",
      "You will return the answer one element: \"the correct label\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Context: The modern Conservative Party was created out of the 'Pittite' Tories of the early 19th century. In the late 1820s disputes over political reform broke up this grouping. A government led by the Duke of Wellington collapsed amidst dire election results. Following this disaster Robert Peel set about assembling a new coalition of forces. Peel issued the Tamworth Manifesto in 1834 which set out the basic principles of Conservatism; – the necessity in specific cases of reform in order to survive, but an opposition to unnecessary change, that could lead to 'a perpetual vortex of agitation'. Meanwhile, the Whigs, along with free trade Tory followers of Robert Peel, and independent Radicals, formed the Liberal Party under Lord Palmerston in 1859, and transformed into a party of the growing urban middle-class, under the long leadership of William Ewart Gladstone.\n",
      "Question: What was created by the modern Conservative Party in 1859 to define basic Conservative principles?\n",
      "Answer: unanswerable\n",
      "\n",
      "Context: Carnival means weeks of events that bring colourfully decorated floats, contagiously throbbing music, luxuriously costumed groups of celebrants of all ages, King and Queen elections, electrifying jump-ups and torchlight parades, the Jouvert morning: the Children's Parades and finally the Grand Parade. Aruba's biggest celebration is a month-long affair consisting of festive 'jump-ups' (street parades), spectacular parades and creative contests. Music and flamboyant costumes play a central role, from the Queen elections to the Grand Parade. Street parades continue in various districts throughout the month, with brass band, steel drum and roadmarch tunes. On the evening before Lent, Carnival ends with the symbolic burning of King Momo.\n",
      "Question: When is King Mom symbolically burnt?\n",
      "Answer: On the evening before Lent\n",
      "\n",
      "Context: The Alps are a crescent shaped geographic feature of central Europe that ranges in a 800 km (500 mi) arc from east to west and is 200 km (120 mi) in width. The mean height of the mountain peaks is 2.5 km (1.6 mi). The range stretches from the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland. The range continues toward Vienna in Austria, and east to the Adriatic Sea and into Slovenia. To the south it dips into northern Italy and to the north extends to the south border of Bavaria in Germany. In areas like Chiasso, Switzerland, and Neuschwanstein, Bavaria, the demarcation between the mountain range and the flatlands are clear; in other places such as Geneva, the demarcation is less clear. The countries with the greatest alpine territory are Switzerland, France, Austria and Italy.\n",
      "Question: How far does the Alps range stretch?\n",
      "Answer: the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Context: The final major evolution of the steam engine design was the use of steam turbines starting in the late part of the 19th century. Steam turbines are generally more efficient than reciprocating piston type steam engines (for outputs above several hundred horsepower), have fewer moving parts, and provide rotary power directly instead of through a connecting rod system or similar means. Steam turbines virtually replaced reciprocating engines in electricity generating stations early in the 20th century, where their efficiency, higher speed appropriate to generator service, and smooth rotation were advantages. Today most electric power is provided by steam turbines. In the United States 90% of the electric power is produced in this way using a variety of heat sources. Steam turbines were extensively applied for propulsion of large ships throughout most of the 20th century.\n",
      "Question: Most power of what sort is generated by steam turbines today?\n",
      "Answer: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/squad_v2/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dd703025-54d8-4349-b0d6-736d2380e966",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 276,
     "referenced_widgets": [
      "46432161e76f4bb38de65858971a6430",
      "3885fbcb1a0f4c05aad284705b71c797"
     ]
    },
    "executionInfo": {
     "elapsed": 81023,
     "status": "ok",
     "timestamp": 1686855741117,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "dd703025-54d8-4349-b0d6-736d2380e966",
    "outputId": "959eb486-adea-4139-bae9-a1a2ff377ac0"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46432161e76f4bb38de65858971a6430",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:langchain.chat_models.openai:Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88154 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "WARNING:langchain.chat_models.openai:Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88629 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "WARNING:langchain.chat_models.openai:Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88219 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "WARNING:langchain.chat_models.openai:Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88915 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n",
      "WARNING:langchain.chat_models.openai:Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo in organization org-etZVkYhAIYGmLcxLmarMmAPo on tokens per min. Limit: 90000 / min. Current: 88534 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Cost: 0.1792\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> f1     </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7019 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.59     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mf1    \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7019\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.59    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef51ce12",
   "metadata": {
    "id": "ef51ce12"
   },
   "source": [
    "We are at 59% accuracy when labeling the first 100 examples. Let's see if we can use confidence scores to improve accuracy further by removing the less confident examples from our labeled set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d7645ab",
   "metadata": {
    "id": "4d7645ab"
   },
   "source": [
    "### Compute confidence scores\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4aa4e28a",
   "metadata": {
    "executionInfo": {
     "elapsed": 146,
     "status": "ok",
     "timestamp": 1686861679354,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "4aa4e28a"
   },
   "outputs": [],
   "source": [
    "# Start computing confidence scores (using Refuel's LLMs)\n",
    "os.environ[\"REFUEL_API_KEY\"] = \"sk-xxxxxxxxxxxxxxxx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5fbc1264",
   "metadata": {
    "executionInfo": {
     "elapsed": 198,
     "status": "ok",
     "timestamp": 1686861680521,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "5fbc1264"
   },
   "outputs": [],
   "source": [
    "config[\"model\"][\"compute_confidence\"] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1998f5e4",
   "metadata": {
    "executionInfo": {
     "elapsed": 179,
     "status": "ok",
     "timestamp": 1686861681560,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "1998f5e4"
   },
   "outputs": [],
   "source": [
    "agent = LabelingAgent(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "119e6f22",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 580,
     "referenced_widgets": [
      "b8282a45d3eb4aecab56744e1472ee6d",
      "50c5bb31a96841b89dcaee466a056d8c"
     ]
    },
    "executionInfo": {
     "elapsed": 1680,
     "status": "ok",
     "timestamp": 1686861684444,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "119e6f22",
    "outputId": "9837fb0d-f77e-4634-de0a-15cf586e9272"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8282a45d3eb4aecab56744e1472ee6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┌──────────────────────────┬─────────┐\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Total Estimated Cost     </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $7.5646 </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Number of Examples       </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> 2000    </span>│\n",
       "│<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\"> Average cost per example </span>│<span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\"> $0.0038 </span>│\n",
       "└──────────────────────────┴─────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┌──────────────────────────┬─────────┐\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mTotal Estimated Cost    \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$7.5646\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mNumber of Examples      \u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m2000   \u001b[0m\u001b[1;32m \u001b[0m│\n",
       "│\u001b[1;35m \u001b[0m\u001b[1;35mAverage cost per example\u001b[0m\u001b[1;35m \u001b[0m│\u001b[1;32m \u001b[0m\u001b[1;32m$0.0038\u001b[0m\u001b[1;32m \u001b[0m│\n",
       "└──────────────────────────┴─────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────── </span>Prompt Example<span style=\"color: #00ff00; text-decoration-color: #00ff00\"> ──────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────── \u001b[0mPrompt Example\u001b[92m ──────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert at answering questions based on wikipedia articles. Your job is to answer the following questions using the context provided with the question. The answer is a continuous span of words from the context. Use the context to answer the question. If the question cannot be answered using the context, answer the question as unanswerable.\n",
      "\n",
      "You will return the answer one element: \"the correct label\"\n",
      "\n",
      "\n",
      "Some examples with their output answers are provided below:\n",
      "\n",
      "Context: The modern Conservative Party was created out of the 'Pittite' Tories of the early 19th century. In the late 1820s disputes over political reform broke up this grouping. A government led by the Duke of Wellington collapsed amidst dire election results. Following this disaster Robert Peel set about assembling a new coalition of forces. Peel issued the Tamworth Manifesto in 1834 which set out the basic principles of Conservatism; – the necessity in specific cases of reform in order to survive, but an opposition to unnecessary change, that could lead to 'a perpetual vortex of agitation'. Meanwhile, the Whigs, along with free trade Tory followers of Robert Peel, and independent Radicals, formed the Liberal Party under Lord Palmerston in 1859, and transformed into a party of the growing urban middle-class, under the long leadership of William Ewart Gladstone.\n",
      "Question: What was created by the modern Conservative Party in 1859 to define basic Conservative principles?\n",
      "Answer: unanswerable\n",
      "\n",
      "Context: Carnival means weeks of events that bring colourfully decorated floats, contagiously throbbing music, luxuriously costumed groups of celebrants of all ages, King and Queen elections, electrifying jump-ups and torchlight parades, the Jouvert morning: the Children's Parades and finally the Grand Parade. Aruba's biggest celebration is a month-long affair consisting of festive 'jump-ups' (street parades), spectacular parades and creative contests. Music and flamboyant costumes play a central role, from the Queen elections to the Grand Parade. Street parades continue in various districts throughout the month, with brass band, steel drum and roadmarch tunes. On the evening before Lent, Carnival ends with the symbolic burning of King Momo.\n",
      "Question: When is King Mom symbolically burnt?\n",
      "Answer: On the evening before Lent\n",
      "\n",
      "Context: The Alps are a crescent shaped geographic feature of central Europe that ranges in a 800 km (500 mi) arc from east to west and is 200 km (120 mi) in width. The mean height of the mountain peaks is 2.5 km (1.6 mi). The range stretches from the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland. The range continues toward Vienna in Austria, and east to the Adriatic Sea and into Slovenia. To the south it dips into northern Italy and to the north extends to the south border of Bavaria in Germany. In areas like Chiasso, Switzerland, and Neuschwanstein, Bavaria, the demarcation between the mountain range and the flatlands are clear; in other places such as Geneva, the demarcation is less clear. The countries with the greatest alpine territory are Switzerland, France, Austria and Italy.\n",
      "Question: How far does the Alps range stretch?\n",
      "Answer: the Mediterranean Sea north above the Po basin, extending through France from Grenoble, eastward through mid and southern Switzerland\n",
      "\n",
      "Now I want you to label the following example:\n",
      "Context: The final major evolution of the steam engine design was the use of steam turbines starting in the late part of the 19th century. Steam turbines are generally more efficient than reciprocating piston type steam engines (for outputs above several hundred horsepower), have fewer moving parts, and provide rotary power directly instead of through a connecting rod system or similar means. Steam turbines virtually replaced reciprocating engines in electricity generating stations early in the 20th century, where their efficiency, higher speed appropriate to generator service, and smooth rotation were advantages. Today most electric power is provided by steam turbines. In the United States 90% of the electric power is produced in this way using a variety of heat sources. Steam turbines were extensively applied for propulsion of large ships throughout most of the 20th century.\n",
      "Question: Most power of what sort is generated by steam turbines today?\n",
      "Answer: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #00ff00; text-decoration-color: #00ff00\">───────────────────────────────────────────────────────────────────────────────────────────────────────────────────</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from autolabel import AutolabelDataset\n",
    "\n",
    "ds = AutolabelDataset(\"data/squad_v2/test.csv\", config=config)\n",
    "agent.plan(ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "63c74705",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 795,
     "referenced_widgets": [
      "d442ea6009ca4add8c72ce808f6a8730",
      "7aa870d8611d4b1c9de1a2ba445b627c"
     ]
    },
    "executionInfo": {
     "elapsed": 218988,
     "status": "ok",
     "timestamp": 1686861903886,
     "user": {
      "displayName": "Abhinav Naikawadi",
      "userId": "14001727525105340618"
     },
     "user_tz": 420
    },
    "id": "63c74705",
    "outputId": "874d8448-8ff2-4871-f25e-f0715cd2afff"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d442ea6009ca4add8c72ce808f6a8730",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
      ],
      "text/plain": []
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Metric: auroc: 0.864\n",
      "Actual Cost: 0.0095\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> f1     </span>┃<span style=\"font-weight: bold\"> support </span>┃<span style=\"font-weight: bold\"> threshold </span>┃<span style=\"font-weight: bold\"> accuracy </span>┃<span style=\"font-weight: bold\"> completion_rate </span>┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7019 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> -inf      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.59     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1       </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9999    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.01            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9892 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 31      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9836    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.31            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9583 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 32      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.979     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9688   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.32            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9596 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 33      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9787    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9697   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.33            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.951  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 34      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9779    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9412   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.34            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9583 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 40      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9635    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.4             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9472 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 41      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9626    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9268   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.41            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9484 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 42      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9472    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9286   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.42            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9264 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 43      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9426    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.907    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.43            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9296 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 45      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9309    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9111   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.45            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9156 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 46      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9124    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8913   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.46            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9191 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 48      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.9082    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8958   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.48            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8949 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 51      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8948    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8431   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.51            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8969 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 52      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.887     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8462   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.52            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8926 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 53      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8836    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8302   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.53            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8946 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 54      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8833    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8333   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.54            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8897 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 55      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8714    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8182   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.55            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8935 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 57      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8684    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8246   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.57            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8781 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 58      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8671    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8103   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.58            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8822 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 60      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8571    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8167   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6             </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.88   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 61      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8523    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8033   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.61            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8819 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 62      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8449    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8065   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.62            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.868  </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 63      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8447    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7937   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.63            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.87   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 64      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8437    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7969   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.64            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8632 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 65      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8363    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7846   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.65            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8653 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 66      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8132    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7879   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.66            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7963 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 72      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7889    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7222   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.72            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.8044 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 75      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7717    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7333   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.75            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7477 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 85      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7305    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6471   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.85            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7506 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 86      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.699     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6512   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.86            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7392 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 88      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6719    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6364   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.88            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7422 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 89      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6664    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6404   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.89            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7183 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 94      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5338    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6064   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.94            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7213 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 95      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.5119    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6105   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.95            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7138 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 96      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.4265    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6042   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.96            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7167 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 97      </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.4058    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.6082   </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.97            </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.7019 </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 100     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.0029    </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 0.59     </span>│<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\"> 1.0             </span>│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mf1    \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1msupport\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mthreshold\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1maccuracy\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mcompletion_rate\u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7019\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m-inf     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.59    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m1.0   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1      \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9999   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.01           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9892\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m31     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9836   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.31           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9583\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m32     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.979    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9688  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.32           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9596\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m33     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9787   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9697  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.33           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.951 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m34     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9779   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9412  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.34           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9583\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m40     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9635   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.95    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.4            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9472\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m41     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9626   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9268  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.41           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9484\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m42     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9472   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9286  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.42           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9264\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m43     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9426   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.907   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.43           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9296\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m45     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9309   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9111  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.45           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9156\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m46     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9124   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8913  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.46           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.9191\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m48     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.9082   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8958  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.48           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8949\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m51     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8948   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8431  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.51           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8969\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m52     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.887    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8462  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.52           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8926\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m53     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8836   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8302  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.53           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8946\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m54     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8833   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8333  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.54           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8897\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m55     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8714   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8182  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.55           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8935\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m57     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8684   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8246  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.57           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8781\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m58     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8671   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8103  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.58           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8822\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m60     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8571   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8167  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.88  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m61     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8523   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8033  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.61           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8819\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m62     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8449   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8065  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.62           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.868 \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m63     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8447   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7937  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.63           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.87  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m64     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8437   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7969  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.64           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8632\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m65     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8363   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7846  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.65           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8653\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m66     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.8132   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7879  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.66           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7963\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m72     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7889   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7222  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.72           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.8044\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m75     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7717   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7333  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.75           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7477\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m85     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.7305   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6471  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.85           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7506\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m86     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.699    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6512  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.86           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7392\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m88     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6719   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6364  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.88           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7422\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m89     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6664   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6404  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.89           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7183\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m94     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5338   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6064  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.94           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7213\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m95     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.5119   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6105  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.95           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7138\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m96     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.4265   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6042  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.96           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7167\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m97     \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.4058   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.6082  \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.97           \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "│\u001b[1;36m \u001b[0m\u001b[1;36m0.7019\u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m100    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.0029   \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m0.59    \u001b[0m\u001b[1;36m \u001b[0m│\u001b[1;36m \u001b[0m\u001b[1;36m1.0            \u001b[0m\u001b[1;36m \u001b[0m│\n",
       "└────────┴─────────┴───────────┴──────────┴─────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Total number of failures: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Total number of failures: \u001b[1;36m0\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds = agent.run(ds, max_items=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abed883b",
   "metadata": {
    "id": "abed883b"
   },
   "source": [
    "Looking at the table above, we can see that if we set the confidence threshold at `0.8449`, we are able to label at 80.65% accuracy and getting a completion rate of 65%. This means, we would ignore all the data points where confidence score is less than `0.8449` (which would end up being around 35% of all samples). This would, however, guarantee a very high quality labeled dataset for us."
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "3885fbcb1a0f4c05aad284705b71c797": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "46432161e76f4bb38de65858971a6430": {
     "model_module": "@jupyter-widgets/output",
     "model_module_version": "1.0.0",
     "model_name": "OutputModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/output",
      "_model_module_version": "1.0.0",
      "_model_name": "OutputModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/output",
      "_view_module_version": "1.0.0",
      "_view_name": "OutputView",
      "layout": "IPY_MODEL_3885fbcb1a0f4c05aad284705b71c797",
      "msg_id": "",
      "outputs": [
       {
        "data": {
         "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #008000; text-decoration-color: #008000\">100/100</span> <span style=\"color: #808000; text-decoration-color: #808000\">0:01:20</span> <span style=\"color: #008080; text-decoration-color: #008080\">0:00:00</span>\nCost in $=0.18, f1=0.7019, support=100, threshold=-inf, accuracy=0.5900, completion_rate=1.0000\n</pre>\n",
         "text/plain": "\u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100/100\u001b[0m \u001b[33m0:01:20\u001b[0m \u001b[36m0:00:00\u001b[0m\nCost in $=0.18, f1=0.7019, support=100, threshold=-inf, accuracy=0.5900, completion_rate=1.0000\n"
        },
        "metadata": {},
        "output_type": "display_data"
       }
      ]
     }
    },
    "50c5bb31a96841b89dcaee466a056d8c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7aa870d8611d4b1c9de1a2ba445b627c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "836e2c02160c4f03903eb0583bb9d700": {
     "model_module": "@jupyter-widgets/output",
     "model_module_version": "1.0.0",
     "model_name": "OutputModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/output",
      "_model_module_version": "1.0.0",
      "_model_name": "OutputModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/output",
      "_view_module_version": "1.0.0",
      "_view_name": "OutputView",
      "layout": "IPY_MODEL_c937612e207b4f80996a0f5265672f59",
      "msg_id": "",
      "outputs": [
       {
        "data": {
         "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Generating Prompts... <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #008000; text-decoration-color: #008000\">100/100</span> <span style=\"color: #808000; text-decoration-color: #808000\">0:00:01</span> <span style=\"color: #008080; text-decoration-color: #008080\">0:00:00</span>\n</pre>\n",
         "text/plain": "Generating Prompts... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100/100\u001b[0m \u001b[33m0:00:01\u001b[0m \u001b[36m0:00:00\u001b[0m\n"
        },
        "metadata": {},
        "output_type": "display_data"
       }
      ]
     }
    },
    "b8282a45d3eb4aecab56744e1472ee6d": {
     "model_module": "@jupyter-widgets/output",
     "model_module_version": "1.0.0",
     "model_name": "OutputModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/output",
      "_model_module_version": "1.0.0",
      "_model_name": "OutputModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/output",
      "_view_module_version": "1.0.0",
      "_view_name": "OutputView",
      "layout": "IPY_MODEL_50c5bb31a96841b89dcaee466a056d8c",
      "msg_id": "",
      "outputs": [
       {
        "data": {
         "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Generating Prompts... <span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #008000; text-decoration-color: #008000\">100/100</span> <span style=\"color: #808000; text-decoration-color: #808000\">0:00:00</span> <span style=\"color: #008080; text-decoration-color: #008080\">0:00:00</span>\n</pre>\n",
         "text/plain": "Generating Prompts... \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100/100\u001b[0m \u001b[33m0:00:00\u001b[0m \u001b[36m0:00:00\u001b[0m\n"
        },
        "metadata": {},
        "output_type": "display_data"
       }
      ]
     }
    },
    "c937612e207b4f80996a0f5265672f59": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d442ea6009ca4add8c72ce808f6a8730": {
     "model_module": "@jupyter-widgets/output",
     "model_module_version": "1.0.0",
     "model_name": "OutputModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/output",
      "_model_module_version": "1.0.0",
      "_model_name": "OutputModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/output",
      "_view_module_version": "1.0.0",
      "_view_name": "OutputView",
      "layout": "IPY_MODEL_7aa870d8611d4b1c9de1a2ba445b627c",
      "msg_id": "",
      "outputs": [
       {
        "data": {
         "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #729c1f; text-decoration-color: #729c1f\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span> <span style=\"color: #008000; text-decoration-color: #008000\">100/100</span> <span style=\"color: #808000; text-decoration-color: #808000\">0:03:38</span> <span style=\"color: #008080; text-decoration-color: #008080\">0:00:00</span>\nCost in $=0.01, f1=0.7019, support=100, threshold=-inf, accuracy=0.5900, completion_rate=1.0000\n</pre>\n",
         "text/plain": "\u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m100/100\u001b[0m \u001b[33m0:03:38\u001b[0m \u001b[36m0:00:00\u001b[0m\nCost in $=0.01, f1=0.7019, support=100, threshold=-inf, accuracy=0.5900, completion_rate=1.0000\n"
        },
        "metadata": {},
        "output_type": "display_data"
       }
      ]
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
